import requests

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.289 Safari/537.36'}

# 发送GET请求获取xhr数据
url = "https://we.51job.com/pc/search?jobArea=110300&keyword=Qt&searchType=2&keywordType="
# url = "https://we.51job.com/api/job/search-pc?api_key=51job&timestamp=1718762010&keyword=Qt&searchType=2&function=&industry=&jobArea=110300&jobArea2=&landmark=&metro=&salary=&workYear=&degree=&companyType=&companySize=&jobType=&issueDate=&sortType=0&pageNum=1&requestId=&keywordType=&pageSize=20&source=1&accountId=&pageCode=sou%7Csou%7Csoulb"
# response = requests.get(url,headers=headers,stream=True, verify=False)
# xhr_data = response.json()  # 将响应数据解析为JSON格式

# 处理获取到的xhr数据
# for data in xhr_data:
#     # 进行数据处理操作
#     pass

# 处理完数据后，可以将结果保存到文件或者输出到控制台
# with open('result.txt', 'w') as f:
#     f.write('Processed data: {}'.format(processed_data))

import requests
from bs4 import BeautifulSoup

# 发送HTTPS请求获取HTML内容
# url = 'https://example.com'
url = 'http://www.ciscoled.com/html/1/%E5%BA%A7%E4%BD%8D%E8%A1%A8.html'
response = requests.get(url)
res = response.raise_for_status()  # 检查请求是否成功
soup = BeautifulSoup(response.text, 'html.parser')

# 解析HTML找到JSON数据的URL
js0 = soup.find('script')
js00 = js0.get('src')
json_url = url + soup.find('script', type='application/ld+json').get('src')

# 发送请求获取JSON数据
json_response = requests.get(json_url)
json_response.raise_for_status()

# 打印JSON数据
print(json_response.json())